{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/qi/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size 1903073\n",
      "Most common words (+UNK) [['UNK', 1196], ('。', 149620), ('\\n', 117070), ('，', 108451), ('、', 19612)]\n",
      "Sample data [1503, 1828, 2, 2, 40, 613, 47, 9, 111, 117] ['潘', '阆', '\\n', '\\n', '酒', '泉', '子', '（', '十', '之']\n",
      "1828 阆 -> 2 \n",
      "\n",
      "1828 阆 -> 1503 潘\n",
      "2 \n",
      " -> 1828 阆\n",
      "2 \n",
      " -> 2 \n",
      "\n",
      "2 \n",
      " -> 2 \n",
      "\n",
      "2 \n",
      " -> 40 酒\n",
      "40 酒 -> 613 泉\n",
      "40 酒 -> 2 \n",
      "\n",
      "WARNING:tensorflow:From <ipython-input-1-16e53679418b>:241: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n",
      "Initialized\n",
      "Average loss at step  0 :  206.421142578125\n",
      "Nearest to 又: 藜, 氅, 煅, 丞, 牛, 其, 颦, 得,\n",
      "Nearest to 月: 戊, 棱, 哉, 凛, 嫣, 宋, 衿, 胜,\n",
      "Nearest to 处: 黻, 上, 况, 僮, 姥, 届, 姑, 可,\n",
      "Nearest to 千: 撞, 淑, 瘢, 鳞, 勍, 骛, 鹁, 歃,\n",
      "Nearest to 有: 瞪, 舫, 嵚, 孀, 赏, 醆, 部, 弧,\n",
      "Nearest to 似: 苑, 蹬, 袭, 笄, 积, 兰, 褓, 筼,\n",
      "Nearest to 明: 搓, 璇, 挥, 吐, 籋, 坠, 膳, 醵,\n",
      "Nearest to 行: 屏, 宸, 镛, 芭, 热, 央, 侯, 甄,\n",
      "Nearest to 如: 钲, 昼, 喘, 鷃, 军, 皋, 窥, 诺,\n",
      "Nearest to 夜: 萦, 孚, 澒, 原, 漆, 雕, 怡, 环,\n",
      "Nearest to 子: 悠, 绀, 装, 毕, 咛, 帘, 绫, 仇,\n",
      "Nearest to 清: 葵, 酴, 堞, 俞, 莎, 耍, 峻, 压,\n",
      "Nearest to 、: 讨, 修, 诙, 袅, 笥, 团, 上, :,\n",
      "Nearest to 。: 愫, 掇, 筱, 辉, 刻, 旷, 毋, 世,\n",
      "Nearest to 生: 熨, 掌, 杲, 绻, 箭, 阃, 及, 伪,\n",
      "Nearest to 柳: 憩, 兵, 帨, 诚, 劈, 渐, 诜, 眺,\n",
      "Average loss at step  2000 :  21.463351945757864\n",
      "Average loss at step  4000 :  5.2879418199062345\n",
      "Average loss at step  6000 :  4.90969002199173\n",
      "Average loss at step  8000 :  4.675899435400963\n",
      "Average loss at step  10000 :  4.610442555099726\n",
      "Nearest to 又:  , 得, 颦, 旗, 丞, 隐, 探, 藜,\n",
      "Nearest to 月: 戊, 嫣, 凛, 哉, 飕, 棱, 蜗, 普,\n",
      "Nearest to 处: 况, 寸, □, 黻, 换, 可, 旦, 真,\n",
      "Nearest to 千: 撞, 鳞, 十,  , 惊, 淑, 岭, 离,\n",
      "Nearest to 有: 赏, 舫, 部, 弧, 缕, 骨, 醆, 意,\n",
      "Nearest to 似: 苑, 袭, 下, 积, 帝, 袅, 兰, 笄,\n",
      "Nearest to 明: 吐, 挥, 璇, 搓, 宗, 坠, 难, 附,\n",
      "Nearest to 行:  , 宸, 芭, 热, 侯, 屏, 煌, 萼,\n",
      "Nearest to 如: 梨, 昼, 喘, 蓦, 皋, 系, 许, 窥,\n",
      "Nearest to 夜: 原, 萦, 怡, 雕, 伸, 迎, 徐, 森,\n",
      "Nearest to 子: 毕, 装, 绫, 臾, 次, 巷, 悠, 致,\n",
      "Nearest to 清: 酴, 堞, 葵, 狂, 常, 峻, 莎, 揾,\n",
      "Nearest to 、: ，, 。, □, 捷, 作, 番, 袅, 栩,\n",
      "Nearest to 。: ，, 、, 必,  , 胪, 番, 鞋, 它,\n",
      "Nearest to 生: 掌, 驱, 熨, 箭, 杲, 极, 退, 咸,\n",
      "Nearest to 柳: 憩, 兵, 摊, □, 渐, 诚, 诜, 传,\n",
      "Average loss at step  12000 :  4.570229660749435\n",
      "Average loss at step  14000 :  4.449028466343879\n",
      "Average loss at step  16000 :  4.467260290741921\n",
      "Average loss at step  18000 :  4.504082497835159\n",
      "Average loss at step  20000 :  4.4408098404407506\n",
      "Nearest to 又: 得, 颦, 丞, 便,  , 旗, 探, 从,\n",
      "Nearest to 月: 戊, 嫣, 凛, 哉, 飕, 衿, 蜗, 普,\n",
      "Nearest to 处: 况, 黻, 寸, 讲, 换, 可, 磨, 真,\n",
      "Nearest to 千: 十, 百, 鳞, 撞, 万, 惊, 离,  ,\n",
      "Nearest to 有: 赏, 舫, 弧, 醆, 部, 嵚, 骨, 瞪,\n",
      "Nearest to 似: 苑, 蹬, 积, 袭, 下, 笄, 向, 表,\n",
      "Nearest to 明: 吐, 璇, 挥, 附, 华, 宗, 滔, 鼙,\n",
      "Nearest to 行:  , 热, 芭, 侯, 宸, 煌, 镛, 萼,\n",
      "Nearest to 如: 梨, 系, 蓦, 钲, 矢, 喘, 毕, 窥,\n",
      "Nearest to 夜: 怡, 原, 雕, 迎, 萦, 孚, 徐, 森,\n",
      "Nearest to 子: 臾, 毕, 装, 次, 溽, 仇, 绀, 骛,\n",
      "Nearest to 清: 酴, 堞, 耍, 狂, 揾, 常, 梓, 葵,\n",
      "Nearest to 、: ，, 。, 番, 乖, 捷, 缉, 都, 忻,\n",
      "Nearest to 。: ，, 、, \n",
      ", 嫣, 鹧, 罨, 愫, ）,\n",
      "Nearest to 生: 掌, 箭, 驱, 熨, 杲, 退, 原, 阃,\n",
      "Nearest to 柳: 摊, 兵, 憩, 渐, 诜, 称, 沙, 诚,\n",
      "Average loss at step  22000 :  4.389726684808731\n",
      "Average loss at step  24000 :  4.345453436493874\n",
      "Average loss at step  26000 :  4.365768854737282\n",
      "Average loss at step  28000 :  4.40806191778183\n",
      "Average loss at step  30000 :  4.320242966771126\n",
      "Nearest to 又: 便, 得, 丞, 旗, 颦,  , 氅, 揩,\n",
      "Nearest to 月: 戊, 嫣, 哉, 飕, 凛, 蜗, 衿, 普,\n",
      "Nearest to 处: 况, 囗, 黻, 寸, 磨, 罢, 讲, 姥,\n",
      "Nearest to 千: 百, 十, 万, 囗, 鳞, 撞, 离,  ,\n",
      "Nearest to 有: 赏, 嵚, 醆, 舫, 部, 丞, 得, 瞪,\n",
      "Nearest to 似: 如, 苑, 向, 蹬, 下, 表, 袭, 伐,\n",
      "Nearest to 明: 吐, 璇, 华, 附, 滔, 洎, 鼙, 疲,\n",
      "Nearest to 行:  , 热, 煌, 侯, 芭, 邸, 甄, 匪,\n",
      "Nearest to 如: 似, 梨, 系, 毕, 钲, 囗, 喘, 矢,\n",
      "Nearest to 夜: 囗, 原, 怡, 雕, 邱, 森, 澒, 拱,\n",
      "Nearest to 子: 臾, 次, 毕, 装, 溽, 仇, 囗, 绀,\n",
      "Nearest to 清: 堞, 狂, 揾, 耍, , 酴, 常, 泻,\n",
      "Nearest to 、: ，, 。, 囗, 番, 簿,  , 话, 栩,\n",
      "Nearest to 。: ，, 、, \n",
      ", 愫, 镝, ）, 鹧, 胪,\n",
      "Nearest to 生: 掌, 箭, 驱, 熨, 杲, 原, 退, 阃,\n",
      "Nearest to 柳: 摊, 兵, 憩, 渐, 称, 娇, 沙, 诜,\n",
      "Average loss at step  32000 :  4.164133338809013\n",
      "Average loss at step  34000 :  4.201560739755631\n",
      "Average loss at step  36000 :  4.22954422044754\n",
      "Average loss at step  38000 :  4.189819645404816\n",
      "Average loss at step  40000 :  4.183247787415981\n",
      "Nearest to 又: 便, 旗, 颦, 得, 肖, 揩, □, 氅,\n",
      "Nearest to 月: 戊, 飕, 哉, 嫣, 凛, 参, 巴, 衿,\n",
      "Nearest to 处: 况, 囗, 黻, 磨, 罢, □, 寸, 偶,\n",
      "Nearest to 千: 万, 百, 十, 囗, 鳞, 撞, 九, 离,\n",
      "Nearest to 有: 嵚, 赏, 醆, 部, 舫, 仿, 丞, 是,\n",
      "Nearest to 似: 如, 向, 苑, 下, 表, 蹬, 袭, 讯,\n",
      "Nearest to 明: 吐, 璇, 华, 附, 洎, 滔, 鼙, 婺,\n",
      "Nearest to 行:  , 煌, 热, 导, 匪, 捱, 邸, 侯,\n",
      "Nearest to 如: 似, 系, 毕, 囗, 梨, 蓦, 钲, 喘,\n",
      "Nearest to 夜: 囗, 怡, 原, 邱, 雕, 姊, 徐, 笛,\n",
      "Nearest to 子: 次, 臾, 毕, 溽, 仇, 绀, 寅, 装,\n",
      "Nearest to 清: 狂, 耍, , 堞, 揾, 常, 梓, 泻,\n",
      "Nearest to 、: ，, 。, 囗, 缉, 濠, 栩, 乖, 番,\n",
      "Nearest to 。: ，, 、, 资, 菩, 仞, 镝, 它, },\n",
      "Nearest to 生: 掌, 驱, 熨, 原, 杲, 退, 箭, 析,\n",
      "Nearest to 柳: 摊, 兵, 娇, 竹, 渐, 憩, 渍, 沙,\n",
      "Average loss at step  42000 :  4.213341261863708\n",
      "Average loss at step  44000 :  4.20055575978756\n",
      "Average loss at step  46000 :  4.236240658402443\n",
      "Average loss at step  48000 :  4.28048623073101\n",
      "Average loss at step  50000 :  4.257828953623772\n",
      "Nearest to 又: 便, 颦, 旗, 揩, 肖, 褭, 得, 骅,\n",
      "Nearest to 月: 戊, 飕, 哉, 巴, 嫣, 凛, 衿, 日,\n",
      "Nearest to 处: 况, 囗, 黻, 磨, 罢, 讲, 卵, 姥,\n",
      "Nearest to 千: 百, 万, 十, 囗, 鳞, 九, 离, 赞,\n",
      "Nearest to 有: 醆, 嵚, 赏, 部, 仿, 舫, 是, 丞,\n",
      "Nearest to 似: 如, 苑, 向, 下, 表, 讯, 蹬, 伐,\n",
      "Nearest to 明: 吐, 璇, 华, 鼙, 疲, 洎, 滔, 附,\n",
      "Nearest to 行:  , 热, 煌, 捱, 邸, 匪, 导, 甄,\n",
      "Nearest to 如: 似, 系, 毕, 钲, 嶂, 囗, 蓦, 梨,\n",
      "Nearest to 夜: 囗, 怡, 邱, 更, 澒, 雕, 徐, 原,\n",
      "Nearest to 子: 臾, 次, 寅, 毕, 溽, 绀, 仇, 装,\n",
      "Nearest to 清: 狂, 耍, , 揾, 堞, 梓, 剔, 常,\n",
      "Nearest to 、: ，, 。, 囗, 番, 缉, 濠, 乖, 肋,\n",
      "Nearest to 。: ，, 、, \n",
      ", ）, 鹧, 罨, 愫, 囗,\n",
      "Nearest to 生: 掌, 原, 驱, 熨, 箭, 退, 杲, 析,\n",
      "Nearest to 柳: 摊, 兵, 娇, 渍, 竹, 渐, 沙, 准,\n",
      "Average loss at step  52000 :  4.240535513401031\n",
      "Average loss at step  54000 :  4.196116251826286\n",
      "Average loss at step  56000 :  4.236288044452667\n",
      "Average loss at step  58000 :  4.25362982904911\n",
      "Average loss at step  60000 :  4.176006798148155\n",
      "Nearest to 又: 便, 旗, 得, 颦, 蓓, 肖, 来, 骅,\n",
      "Nearest to 月: 戊, 飕, 嫣, 日, 秋, 清, 哉, 分,\n",
      "Nearest to 处: 况, 罢, 囗, 黻, 磨, 姥, 讲, 卵,\n",
      "Nearest to 千: 万, 百, 十, 囗, 鳞, 九, 离, 赞,\n",
      "Nearest to 有: 嵚, 是, 醆, 赏, 部, 仿, 得, 犒,\n",
      "Nearest to 似: 如, 向, 表, 下, 苑, 蹬, 伐, 是,\n",
      "Nearest to 明: 吐, 璇, 华, 滔, 鼙, 附, 婺, 蝇,\n",
      "Nearest to 行: 热,  , 邸, 煌, 导, 捱, 匪, 昴,\n",
      "Nearest to 如: 似, 毕, 系, 梨, 嶂, 钲, 喘, 掐,\n",
      "Nearest to 夜: 囗, 邱, 更, 怡, 姊, 澒, 雕, 森,\n",
      "Nearest to 子: 臾, 次, 溽, 寅, 仇, 簉, 毕, 咏,\n",
      "Nearest to 清: 狂, , 揾, 耍, 月, 堞, 蓠, 剔,\n",
      "Nearest to 、: ，, 。, 囗, 番, 颙, 偿, 栩, 簿,\n",
      "Nearest to 。: ，, 、, \n",
      ", ）, 镝, 愫, 胪, 嫣,\n",
      "Nearest to 生: 掌, 熨, 驱, 原, 箭, 析, 退, 杲,\n",
      "Nearest to 柳: 摊, 娇, 花, 渍, 兵, 竹, 沙, 渐,\n",
      "Average loss at step  62000 :  4.09940416097641\n",
      "Average loss at step  64000 :  4.113055974721909\n",
      "Average loss at step  66000 :  4.1270759488344195\n",
      "Average loss at step  68000 :  4.108153470396996\n",
      "Average loss at step  70000 :  4.106202446818352\n",
      "Nearest to 又: 便, 旗, 肖, 蓓, 颦, 俱, 褭, 骅,\n",
      "Nearest to 月: 戊, 飕, 日, 哉, 巴, 嫣, 秋, 参,\n",
      "Nearest to 处: 囗, 罢, 况, 磨, 黻, 卵, 姥, 讲,\n",
      "Nearest to 千: 万, 百, 十, 囗, 九, 鳞, 赞, 离,\n",
      "Nearest to 有: 是, 嵚, 醆, 赏, 仿, 部, 犒, 得,\n",
      "Nearest to 似: 如, 表, 向, 下, 苑, 与, 是, 蹬,\n",
      "Nearest to 明: 吐, 璇, 华, 洎, 附, 滔, 鼙, 诱,\n",
      "Nearest to 行: 煌, 导, 捱, 热, 邸, 匪, 昴, 骀,\n",
      "Nearest to 如: 似, 系, 毕, 嶂, 蓦, 喘, 梨, 钲,\n",
      "Nearest to 夜: 囗, 宵, 邱, 更, 怡, 姊, 徐, 雕,\n",
      "Nearest to 子: 次, 臾, 溽, 寅, 仇, 毕, 咏, 嵬,\n",
      "Nearest to 清: 狂, , 耍, 揾, 月, 泻, 堞, 乐,\n",
      "Nearest to 、: ，, 。, 囗, 缉, 濠, 乖, 俱, 栩,\n",
      "Nearest to 。: ，, 、, 菩, ）, 资, 镝, 胪, \n",
      ",\n",
      "Nearest to 生: 掌, 原, 析, 退, 熨, 驱, 杲, 箭,\n",
      "Nearest to 柳: 摊, 娇, 竹, 花, 杨, 渍, 兵, 低,\n",
      "Average loss at step  72000 :  4.126074243068695\n",
      "Average loss at step  74000 :  4.125884716033935\n",
      "Average loss at step  76000 :  4.1946677457094195\n",
      "Average loss at step  78000 :  4.197617957949638\n",
      "Average loss at step  80000 :  4.191707204818726\n",
      "Nearest to 又: 便, 褭, 肖, 骅, 已, 旗, 也, 还,\n",
      "Nearest to 月: 戊, 飕, 日, 秋, 巴, 影, 醥, 衿,\n",
      "Nearest to 处: 囗, 磨, 罢, 黻, 况, 讲, 卵, 姥,\n",
      "Nearest to 千: 万, 百, 十, 囗, 九, 鳞, 赞, 离,\n",
      "Nearest to 有: 醆, 是, 嵚, 赏, 犒, 仿, 富, 部,\n",
      "Nearest to 似: 如, 向, 表, 苑, 下, 讯, 改, 为,\n",
      "Nearest to 明: 吐, 璇, 华, 鼙, 滔, 洎, 疲, 附,\n",
      "Nearest to 行: 捱, 煌, 邸, 箵, 热, 导, 给, 匪,\n",
      "Nearest to 如: 似, 系, 毕, 嶂, 掐, 钲, 矢, 蓦,\n",
      "Nearest to 夜: 囗, 邱, 宵, 槛, 怡, 更, 徐, 澒,\n",
      "Nearest to 子: 寅, 次, 臾, 咏, 嵬, 溽, 戌, 绀,\n",
      "Nearest to 清: 狂, , 耍, 揾, 月, 剔, 蓠, 梓,\n",
      "Nearest to 、: ，, 。, 囗, 番, 濠, 缉, 怜, 肋,\n",
      "Nearest to 。: ，, 、, \n",
      ", 愫, ）, 鹧, 镝, 罨,\n",
      "Nearest to 生: 掌, 原, 熨, 析, 退, 驱, 箭, 杲,\n",
      "Nearest to 柳: 摊, 竹, 杨, 娇, 渍, 花, 兵, 戮,\n",
      "Average loss at step  82000 :  4.170699589252472\n",
      "Average loss at step  84000 :  4.137689306259155\n",
      "Average loss at step  86000 :  4.186060482621193\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average loss at step  88000 :  4.174636708736419\n",
      "Average loss at step  90000 :  4.097789842367172\n",
      "Nearest to 又: 便, 蓓, 也, 肖, 骅, 已, 褭, 旗,\n",
      "Nearest to 月: 戊, 日, 飕, 清, 秋, 嫣, 影, 醥,\n",
      "Nearest to 处: 罢, 黻, 囗, 况, 磨, 曾, 卵, 姥,\n",
      "Nearest to 千: 万, 百, 十, 囗, 九, 鳞, 躇, 赞,\n",
      "Nearest to 有: 是, 醆, 嵚, 得, 犒, 无, 见, 赏,\n",
      "Nearest to 似: 如, 表, 向, 是, 为, 下, 改, 蹬,\n",
      "Nearest to 明: 吐, 璇, 圆, 华, 蝇, 滔, 婺, 光,\n",
      "Nearest to 行: 邸, 导, 煌, 捱, 箵, 给, 五, 热,\n",
      "Nearest to 如: 似, 毕, 系, 嶂, 梨, 掐, 钲, 喘,\n",
      "Nearest to 夜: 囗, 宵, 更, 槛, 邱, 晓, 姊, 徐,\n",
      "Nearest to 子: 寅, 次, 臾, 戌, 溽, 咏, 嵬, 仇,\n",
      "Nearest to 清: 狂, , 揾, 耍, 月, 蓠, 剔, 堞,\n",
      "Nearest to 、: ，, 。, 囗, 濠, 肋, 颙, 添, 怜,\n",
      "Nearest to 。: ，, 、, \n",
      ", 愫, 菩, ）, 鸪, 鹧,\n",
      "Nearest to 生: 掌, 原, 熨, 驱, 退, 析, 箭, 董,\n",
      "Nearest to 柳: 杨, 花, 娇, 摊, 竹, 渍, 栽, 戮,\n",
      "Average loss at step  92000 :  4.047216898918152\n",
      "Average loss at step  94000 :  4.087995460629463\n",
      "Average loss at step  96000 :  4.05107104742527\n",
      "Average loss at step  98000 :  4.080057385444641\n",
      "Average loss at step  100000 :  4.058966382980347\n",
      "Nearest to 又: 便, 肖, 俱, 蓓, 已, 旗, 还, 早,\n",
      "Nearest to 月: 戊, 飕, 日, 秋, 醥, 巴, 影, 衿,\n",
      "Nearest to 处: 囗, 罢, 磨, 意, 况, 黻, 曾, 央,\n",
      "Nearest to 千: 万, 百, 十, 囗, 九, 鳞, 离, 赞,\n",
      "Nearest to 有: 是, 嵚, 醆, 在, 见, 犒, 得, 仿,\n",
      "Nearest to 似: 如, 表, 是, 向, 为, 在, 与, 下,\n",
      "Nearest to 明: 吐, 璇, 圆, 光, 洎, 诱, 华, 蝇,\n",
      "Nearest to 行: 导, 煌, 邸, 箵, 捱, 骀, 谱, 五,\n",
      "Nearest to 如: 似, 毕, 系, 嶂, 。, 喘, 胞, 梨,\n",
      "Nearest to 夜: 囗, 宵, 邱, 晓, 徐, 更, 槛, 姊,\n",
      "Nearest to 子: 次, 寅, 戌, 仇, 臾, 溽, 濑, 嵬,\n",
      "Nearest to 清: 狂, , 耍, 揾, 泻, 剔, 蓠, 中,\n",
      "Nearest to 、: ，, 。, 囗, 濠, 缉, 俱, 颙, 泸,\n",
      "Nearest to 。: ，, 、, 菩, \n",
      ", ）, 筲, 需, 鹧,\n",
      "Nearest to 生: 原, 掌, 析, 退, 熨, 驱, 杲, 向,\n",
      "Nearest to 柳: 杨, 竹, 花, 娇, 渍, 摊, 栽, 苹,\n"
     ]
    }
   ],
   "source": [
    "# Copyright 2015 The TensorFlow Authors. All Rights Reserved.\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     http://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License.\n",
    "# ==============================================================================\n",
    "\"\"\"Basic word2vec example.\"\"\"\n",
    "\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import collections\n",
    "import math\n",
    "import os\n",
    "import sys\n",
    "import argparse\n",
    "import random\n",
    "from tempfile import gettempdir\n",
    "import zipfile\n",
    "\n",
    "import numpy as np\n",
    "from six.moves import urllib\n",
    "from six.moves import xrange  # pylint: disable=redefined-builtin\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.contrib.tensorboard.plugins import projector\n",
    "\n",
    "# Give a folder path as an argument with '--log_dir' to save\n",
    "# TensorBoard summaries. Default is a log folder in current directory.\n",
    "current_path = os.path.dirname(os.path.realpath(sys.argv[0]))\n",
    "\n",
    "parser = argparse.ArgumentParser()\n",
    "parser.add_argument(\n",
    "    '--log_dir',\n",
    "    type=str,\n",
    "    default=os.path.join(current_path, 'log'),\n",
    "    help='The log directory for TensorBoard summaries.')\n",
    "FLAGS, unparsed = parser.parse_known_args()\n",
    "\n",
    "# Create the directory for TensorBoard variables if there is not.\n",
    "if not os.path.exists(FLAGS.log_dir):\n",
    "  os.makedirs(FLAGS.log_dir)\n",
    "\n",
    "# Step 1: Download the data.\n",
    "# url = 'http://mattmahoney.net/dc/'\n",
    "\n",
    "\n",
    "# # pylint: disable=redefined-outer-name\n",
    "# def maybe_download(filename, expected_bytes):\n",
    "#   \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n",
    "#   local_filename = os.path.join(gettempdir(), filename)\n",
    "#   if not os.path.exists(local_filename):\n",
    "#     local_filename, _ = urllib.request.urlretrieve(url + filename,\n",
    "#                                                    local_filename)\n",
    "#   statinfo = os.stat(local_filename)\n",
    "#   if statinfo.st_size == expected_bytes:\n",
    "#     print('Found and verified', filename)\n",
    "#   else:\n",
    "#     print(statinfo.st_size)\n",
    "#     raise Exception('Failed to verify ' + local_filename +\n",
    "#                     '. Can you get to it with a browser?')\n",
    "#   return local_filename\n",
    "\n",
    "\n",
    "# filename = maybe_download('text8.zip', 31344016)\n",
    "\n",
    "\n",
    "# # Read the data into a list of strings.\n",
    "# def read_data(filename):\n",
    "#   \"\"\"Extract the first file enclosed in a zip file as a list of words.\"\"\"\n",
    "#   with zipfile.ZipFile(filename) as f:\n",
    "#     data = tf.compat.as_str(f.read(f.namelist()[0])).split()\n",
    "#   return data\n",
    "\n",
    "filename = \"QuanSongCi.txt\"\n",
    "# Read the data into a list of strings.\n",
    "def read_data(filename):\n",
    "    with open(filename, encoding=\"utf-8\") as f:\n",
    "        data = f.read()\n",
    "    data = list(data)\n",
    "    return data\n",
    "\n",
    "vocabulary = read_data(filename)\n",
    "print('Data size', len(vocabulary))\n",
    "\n",
    "# Step 2: Build the dictionary and replace rare words with UNK token.\n",
    "vocabulary_size = 5000\n",
    "\n",
    "\n",
    "def build_dataset(words, n_words):\n",
    "  \"\"\"Process raw inputs into a dataset.\"\"\"\n",
    "  count = [['UNK', -1]]\n",
    "  count.extend(collections.Counter(words).most_common(n_words - 1))\n",
    "  dictionary = dict()\n",
    "  for word, _ in count:\n",
    "    dictionary[word] = len(dictionary)\n",
    "  data = list()\n",
    "  unk_count = 0\n",
    "  for word in words:\n",
    "    index = dictionary.get(word, 0)\n",
    "    if index == 0:  # dictionary['UNK']\n",
    "      unk_count += 1\n",
    "    data.append(index)\n",
    "  count[0][1] = unk_count\n",
    "  reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "  return data, count, dictionary, reversed_dictionary\n",
    "\n",
    "\n",
    "# Filling 4 global variables:\n",
    "# data - list of codes (integers from 0 to vocabulary_size-1).\n",
    "#   This is the original text but words are replaced by their codes\n",
    "# count - map of words(strings) to count of occurrences\n",
    "# dictionary - map of words(strings) to their codes(integers)\n",
    "# reverse_dictionary - maps codes(integers) to words(strings)\n",
    "data, count, dictionary, reverse_dictionary = build_dataset(\n",
    "    vocabulary, vocabulary_size)\n",
    "del vocabulary  # Hint to reduce memory.\n",
    "print('Most common words (+UNK)', count[:5])\n",
    "print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])\n",
    "\n",
    "\n",
    "import json\n",
    "with open(\"dictionary.json\", \"w\", encoding='utf-8') as f1:\n",
    "    f1.write(json.dumps(dictionary))\n",
    "with open(\"reverse_dictionary.json\", \"w\", encoding='utf-8') as f2:\n",
    "    f2.write(json.dumps(reverse_dictionary))\n",
    "\n",
    "data_index = 0\n",
    "\n",
    "\n",
    "# Step 3: Function to generate a training batch for the skip-gram model.\n",
    "def generate_batch(batch_size, num_skips, skip_window):\n",
    "  global data_index\n",
    "  assert batch_size % num_skips == 0\n",
    "  assert num_skips <= 2 * skip_window\n",
    "  batch = np.ndarray(shape=(batch_size), dtype=np.int32)\n",
    "  labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)\n",
    "  span = 2 * skip_window + 1  # [ skip_window target skip_window ]\n",
    "  buffer = collections.deque(maxlen=span)  # pylint: disable=redefined-builtin\n",
    "  if data_index + span > len(data):\n",
    "    data_index = 0\n",
    "  buffer.extend(data[data_index:data_index + span])\n",
    "  data_index += span\n",
    "  for i in range(batch_size // num_skips):\n",
    "    context_words = [w for w in range(span) if w != skip_window]\n",
    "    words_to_use = random.sample(context_words, num_skips)\n",
    "    for j, context_word in enumerate(words_to_use):\n",
    "      batch[i * num_skips + j] = buffer[skip_window]\n",
    "      labels[i * num_skips + j, 0] = buffer[context_word]\n",
    "    if data_index == len(data):\n",
    "      buffer.extend(data[0:span])\n",
    "      data_index = span\n",
    "    else:\n",
    "      buffer.append(data[data_index])\n",
    "      data_index += 1\n",
    "  # Backtrack a little bit to avoid skipping words in the end of a batch\n",
    "  data_index = (data_index + len(data) - span) % len(data)\n",
    "  return batch, labels\n",
    "\n",
    "\n",
    "batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)\n",
    "for i in range(8):\n",
    "  print(batch[i], reverse_dictionary[batch[i]], '->', labels[i, 0],\n",
    "        reverse_dictionary[labels[i, 0]])\n",
    "\n",
    "# Step 4: Build and train a skip-gram model.\n",
    "\n",
    "batch_size = 128\n",
    "embedding_size = 128  # Dimension of the embedding vector.\n",
    "skip_window = 1  # How many words to consider left and right.\n",
    "num_skips = 2  # How many times to reuse an input to generate a label.\n",
    "num_sampled = 64  # Number of negative examples to sample.\n",
    "\n",
    "# We pick a random validation set to sample nearest neighbors. Here we limit the\n",
    "# validation samples to the words that have a low numeric ID, which by\n",
    "# construction are also the most frequent. These 3 variables are used only for\n",
    "# displaying model accuracy, they don't affect calculation.\n",
    "valid_size = 16  # Random set of words to evaluate similarity on.\n",
    "valid_window = 100  # Only pick dev samples in the head of the distribution.\n",
    "valid_examples = np.random.choice(valid_window, valid_size, replace=False)\n",
    "\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data.\n",
    "  with tf.name_scope('inputs'):\n",
    "    train_inputs = tf.placeholder(tf.int32, shape=[batch_size])\n",
    "    train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])\n",
    "    valid_dataset = tf.constant(valid_examples, dtype=tf.int32)\n",
    "\n",
    "  # Ops and variables pinned to the CPU because of missing GPU implementation\n",
    "  with tf.device('/cpu:0'):\n",
    "    # Look up embeddings for inputs.\n",
    "    with tf.name_scope('embeddings'):\n",
    "      embeddings = tf.Variable(\n",
    "          tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))\n",
    "      embed = tf.nn.embedding_lookup(embeddings, train_inputs)\n",
    "\n",
    "    # Construct the variables for the NCE loss\n",
    "    with tf.name_scope('weights'):\n",
    "      nce_weights = tf.Variable(\n",
    "          tf.truncated_normal(\n",
    "              [vocabulary_size, embedding_size],\n",
    "              stddev=1.0 / math.sqrt(embedding_size)))\n",
    "    with tf.name_scope('biases'):\n",
    "      nce_biases = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "\n",
    "  # Compute the average NCE loss for the batch.\n",
    "  # tf.nce_loss automatically draws a new sample of the negative labels each\n",
    "  # time we evaluate the loss.\n",
    "  # Explanation of the meaning of NCE loss:\n",
    "  #   http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/\n",
    "  with tf.name_scope('loss'):\n",
    "    loss = tf.reduce_mean(\n",
    "        tf.nn.nce_loss(\n",
    "            weights=nce_weights,\n",
    "            biases=nce_biases,\n",
    "            labels=train_labels,\n",
    "            inputs=embed,\n",
    "            num_sampled=num_sampled,\n",
    "            num_classes=vocabulary_size))\n",
    "\n",
    "  # Add the loss value as a scalar to summary.\n",
    "  tf.summary.scalar('loss', loss)\n",
    "\n",
    "  # Construct the SGD optimizer using a learning rate of 1.0.\n",
    "  with tf.name_scope('optimizer'):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)\n",
    "\n",
    "  # Compute the cosine similarity between minibatch examples and all embeddings.\n",
    "  norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))\n",
    "  normalized_embeddings = embeddings / norm\n",
    "  valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings,\n",
    "                                            valid_dataset)\n",
    "  similarity = tf.matmul(\n",
    "      valid_embeddings, normalized_embeddings, transpose_b=True)\n",
    "\n",
    "  # Merge all summaries.\n",
    "  merged = tf.summary.merge_all()\n",
    "\n",
    "  # Add variable initializer.\n",
    "  init = tf.global_variables_initializer()\n",
    "\n",
    "  # Create a saver.\n",
    "  saver = tf.train.Saver()\n",
    "\n",
    "# Step 5: Begin training.\n",
    "num_steps = 100001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  # Open a writer to write summaries.\n",
    "  writer = tf.summary.FileWriter(FLAGS.log_dir, session.graph)\n",
    "\n",
    "  # We must initialize all variables before we use them.\n",
    "  init.run()\n",
    "  print('Initialized')\n",
    "\n",
    "  average_loss = 0\n",
    "  for step in xrange(num_steps):\n",
    "    batch_inputs, batch_labels = generate_batch(batch_size, num_skips,\n",
    "                                                skip_window)\n",
    "    feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}\n",
    "\n",
    "    # Define metadata variable.\n",
    "    run_metadata = tf.RunMetadata()\n",
    "\n",
    "    # We perform one update step by evaluating the optimizer op (including it\n",
    "    # in the list of returned values for session.run()\n",
    "    # Also, evaluate the merged op to get all summaries from the returned \"summary\" variable.\n",
    "    # Feed metadata variable to session for visualizing the graph in TensorBoard.\n",
    "    _, summary, loss_val = session.run(\n",
    "        [optimizer, merged, loss],\n",
    "        feed_dict=feed_dict,\n",
    "        run_metadata=run_metadata)\n",
    "    average_loss += loss_val\n",
    "\n",
    "    # Add returned summaries to writer in each step.\n",
    "    writer.add_summary(summary, step)\n",
    "    # Add metadata to visualize the graph for the last run.\n",
    "    if step == (num_steps - 1):\n",
    "      writer.add_run_metadata(run_metadata, 'step%d' % step)\n",
    "\n",
    "    if step % 2000 == 0:\n",
    "      if step > 0:\n",
    "        average_loss /= 2000\n",
    "      # The average loss is an estimate of the loss over the last 2000 batches.\n",
    "      print('Average loss at step ', step, ': ', average_loss)\n",
    "      average_loss = 0\n",
    "\n",
    "    # Note that this is expensive (~20% slowdown if computed every 500 steps)\n",
    "    if step % 10000 == 0:\n",
    "      sim = similarity.eval()\n",
    "      for i in xrange(valid_size):\n",
    "        valid_word = reverse_dictionary[valid_examples[i]]\n",
    "        top_k = 8  # number of nearest neighbors\n",
    "        nearest = (-sim[i, :]).argsort()[1:top_k + 1]\n",
    "        log_str = 'Nearest to %s:' % valid_word\n",
    "        for k in xrange(top_k):\n",
    "          close_word = reverse_dictionary[nearest[k]]\n",
    "          log_str = '%s %s,' % (log_str, close_word)\n",
    "        print(log_str)\n",
    "  final_embeddings = normalized_embeddings.eval()\n",
    "  np.save('embedding.npy', final_embeddings)\n",
    "\n",
    "  # Write corresponding labels for the embeddings.\n",
    "  with open(FLAGS.log_dir + '/metadata.tsv', 'w') as f:\n",
    "    for i in xrange(vocabulary_size):\n",
    "      f.write(reverse_dictionary[i] + '\\n')\n",
    "\n",
    "  # Save the model for checkpoints.\n",
    "  saver.save(session, os.path.join(FLAGS.log_dir, 'model.ckpt'))\n",
    "\n",
    "  # Create a configuration for visualizing embeddings with the labels in TensorBoard.\n",
    "  config = projector.ProjectorConfig()\n",
    "  embedding_conf = config.embeddings.add()\n",
    "  embedding_conf.tensor_name = embeddings.name\n",
    "  embedding_conf.metadata_path = os.path.join(FLAGS.log_dir, 'metadata.tsv')\n",
    "  projector.visualize_embeddings(writer, config)\n",
    "\n",
    "writer.close()\n",
    "\n",
    "# Step 6: Visualize the embeddings.\n",
    "\n",
    "# from matplotlib.font_manager import FontProperties\n",
    "# font = FontProperties(fname='/Library/Fonts/Songti.ttc')\n",
    "\n",
    "# pylint: disable=missing-docstring\n",
    "# Function to draw visualization of distance between embeddings.\n",
    "def plot_with_labels(low_dim_embs, labels, filename):\n",
    "  assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'\n",
    "  plt.figure(figsize=(18, 18))  # in inches\n",
    "  for i, label in enumerate(labels):\n",
    "    x, y = low_dim_embs[i, :]\n",
    "    plt.scatter(x, y)\n",
    "    plt.annotate(\n",
    "        label,\n",
    "        xy=(x, y),\n",
    "        xytext=(5, 2),\n",
    "        textcoords='offset points',\n",
    "        ha='right',\n",
    "        va='bottom')\n",
    "\n",
    "  plt.savefig(filename)\n",
    "\n",
    "\n",
    "try:\n",
    "  # pylint: disable=g-import-not-at-top\n",
    "  from sklearn.manifold import TSNE\n",
    "  import matplotlib.pyplot as plt\n",
    "  from matplotlib.font_manager import _rebuild\n",
    "  _rebuild()\n",
    "  plt.rcParams['font.sans-serif']=[u'SimHei']\n",
    "  plt.rcParams['axes.unicode_minus']=False\n",
    "\n",
    "\n",
    "  tsne = TSNE(\n",
    "      perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact')\n",
    "  plot_only = 500\n",
    "  low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])\n",
    "  labels = [reverse_dictionary[i] for i in xrange(plot_only)]\n",
    "#   plot_with_labels(low_dim_embs, labels, os.path.join(gettempdir(), 'tsne.png'))\n",
    "  plot_with_labels(low_dim_embs, labels, './tsne.png')\n",
    "\n",
    "except ImportError as ex:\n",
    "  print('Please install sklearn, matplotlib, and scipy to show embeddings.')\n",
    "  print(ex)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
